package com.samp.solr.file2solr3;

import com.samp.solr.file2solr.ErrorIntentionExtract;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

/**
 * 生成精排所需要的训练数据。
 * 与1版不同，此版本交叉生成训练数据
 */
public class GenerateTrainingData2 {

//    private static final String SOLR_FIELD_CONTENT = "content";
//    private static final String SOLR_FIELD_INTENTION = "intention";
//    private static final String SOLR_FIELD_CONTENT_FULL_MATCH = "contentFullMatch";
    //    private static final String BASE_SOLR_URL = "http://localhost:8080/solr/callcenter";
    private static final String BASE_SOLR_URL = "http://localhost:8080/solr/gcc-omcp";

//    private static Map<String,Float> weightMap = new HashMap<>();

    private static final Float minScore = 4.0f; //最小分值
    private static final Float minRate = 1.2f; //最小比值


//    static {
//        initialWeightMap();
//    }

    private static void generateData(String rawPath, String outFile,Integer topN) throws Exception{
        Path filePath = Paths.get(rawPath);
        BufferedReader bufferReader = Files.newBufferedReader(filePath, StandardCharsets.UTF_8);
        String line;
        try {
//            Set<String> resultList = new HashSet<>(1000000);
            List<String> resultList = new ArrayList<>(1000000);
//            resultList.add("id\tqid1\tqid2\tquestion1\tquestion2\tis_duplicate");
            SolrClient solrClient =new HttpSolrClient.Builder(BASE_SOLR_URL).build();
            Integer index = 0;
            long start = System.currentTimeMillis();
            long allStart = System.currentTimeMillis();
            Integer count = 0;
            Integer splitNum = 0;
            while( (line = bufferReader.readLine()) != null ){
                if( StringUtils.isBlank(line) ){
                    System.out.println("blank line,continue..");
                    continue;
                }
                String[] splits = line.split("\t");
                if( splits.length != 3 ){
                    System.out.println("error data:"+line);
                    continue;
                }
                count++;
                if( count % 5000 == 0 ){
                    System.out.println("processed 5000 lines, cost="+((System.currentTimeMillis()-start)*1.0/1000)+" 秒,count="+count+",resultSize="+resultList.size());
                    start = System.currentTimeMillis();
                }
                String rawWord = splits[0];
                String rawIntention  = splits[1];
                String intention = GenerateTrainingData.extractIntentionWord(rawIntention);
                if( intention == null ){
                    continue;
                }
                int recordNeed = topN+1;
                List<WordIntention> solrResultList = GenerateTrainingData.getTopNIntentions(solrClient, rawWord, recordNeed);
                if( solrResultList.size() < recordNeed ){
                    recordNeed = solrResultList.size();
                }
                List<WordIntention> dataList = new ArrayList<>();
                for(int i = 0 ; i < recordNeed-1; i++){
                    //如果满足直接返回的条件（分数大于4，且上条记录/当条记录>1.2），则过滤掉。
                    WordIntention wi = solrResultList.get(i);
                    WordIntention wiNext = solrResultList.get(i+1);
                    if( wi.getScore() > minScore && wi.getScore()/wiNext.getScore() > minRate ){
//                        System.out.println("return return.."+line);
                        continue;
                    }
                    dataList.add(wi);
                }
                List<String> transList = generateTrainData(index, dataList);
                resultList.addAll(transList);
                index += transList.size();
                if( resultList.size() > 1000000 ){
                    String temFile = outFile+splitNum+".txt";
                    ErrorIntentionExtract.saveResultToFile(resultList,temFile);
                    resultList.clear();
                    splitNum++;
                }
//                if( index > 100 ){
//                    break;
//                }
            }
//            for(String s:resultList){
//                System.out.println(s);
//            }
            String temFile = outFile+"last.txt";
            ErrorIntentionExtract.saveResultToFile(resultList,temFile);
            System.out.println("save "+ resultList.size()+" lines, cost "+((System.currentTimeMillis()-allStart)*1.0/(1000*60))+" 分钟");
        }catch (IOException e) {
            e.printStackTrace();
        }finally{
            bufferReader.close();
        }
    }

    /**
     * 将语料意图交叉生成训练数据。
     * @param wiList
     * @return
     */
    private static List<String> generateTrainData(Integer index, List<WordIntention> wiList){
        Set<String> intentionSet = new HashSet<>();
        for(WordIntention wi: wiList){
            intentionSet.add(wi.getIntention());
        }
        List<String> resultList = new ArrayList<>();
        for( WordIntention wi : wiList){
            for( String intention : intentionSet){
                Integer isMatch = 0;
                if( wi.getIntention().equals(intention) ){
                    isMatch = 1;
                }
                resultList.add(GenerateTrainingData.getTrainningData(index, wi.getWord(),
                                                GenerateTrainingData.extractIntentionWord(intention), isMatch));
                index++;
            }
        }
        return resultList;
    }

    private static void mergeData(String dirPath, String outFileName) throws Exception{
        String sourceDir = "D:\\gcc-omcp\\new\\zjk\\";
        Path dir = Paths.get(dirPath);
        //通过Files.newDirectoryStream(dir)来遍历文件夹
        List<String> resultList = new ArrayList<>();
        Set<String> set = new HashSet<>();
        Integer index = 0;
        try( DirectoryStream<Path> stream = Files.newDirectoryStream(dir) ){
            for(Path path : stream){
                if( !Files.isDirectory(path) ){
                    String filePath = path.toAbsolutePath().toString();
                    System.out.println("process........"+filePath+", resultSize="+resultList.size());
                    BufferedReader bufferReader = Files.newBufferedReader(path, StandardCharsets.UTF_8);
                    String line;
                    while( (line = bufferReader.readLine()) != null ){
                        String[] splits = line.split("\t");
                        if( splits.length != 6 ){
                            System.out.println("error data:"+line);
                            continue;
                        }
                        String key = splits[3]+splits[4]+splits[5];
                        if( set.contains(key) ){
                            continue;
                        }
                        set.add(key);
                        resultList.add(GenerateTrainingData.getTrainningData(index, splits[3],
                                                splits[4], Integer.valueOf(splits[5])));
                        index++;
//                        if( index > 100 ){
//                            break;
//                        }
                    }
                }
            }
        }catch(Exception e){
            e.printStackTrace();
        }
        System.out.println("result size = "+resultList.size());
        ErrorIntentionExtract.saveResultToFile(resultList,outFileName);
    }

    public static void main(String[] args) {
//        String a = "action.name=查询#target.name=理赔#target.params1=进度";
//        System.out.println(extractIntentionWord(a));

//        String rawPath = "D:\\gcc-omcp\\new\\data\\027_total_words.txt";
//        String outPath = "D:\\gcc-omcp\\trainning\\08\\95511_training_08_";
//        Integer topN = 5;
//        try {
//            generateData(rawPath, outPath, topN);
//        }catch (Exception e){
//            e.printStackTrace();
//        }

          String path = "D:\\gcc-omcp\\trainning\\08\\";
          String outFile = "D:\\gcc-omcp\\trainning\\95511_training_08_merge.txt";
          try {
              mergeData(path, outFile);
          }catch (Exception e){
              e.printStackTrace();
          }


//        List<WordIntention> dataList = new ArrayList<>();
//        dataList.add(new WordIntention("A1", "B1"));
//        dataList.add(new WordIntention("A2", "B2"));
//        dataList.add(new WordIntention("A3", "B1"));
//        dataList.add(new WordIntention("A4", "B3"));
//        dataList.add(new WordIntention("A5", "B1"));
//        List<String> result = generateTrainData(0, dataList);
//        for(String str: result){
//            System.out.println(str);
//        }
    }

}

//class WordIntention2{
//    private String word;
//    private String intention;
//    private Float score;
//
//    public Float getScore() {
//        return score;
//    }
//
//    public void setScore(Float score) {
//        this.score = score;
//    }
//
//    public String getWord() {
//        return word;
//    }
//
//    public void setWord(String word) {
//        this.word = word;
//    }
//
//    public String getIntention() {
//        return intention;
//    }
//
//    public void setIntention(String intention) {
//        this.intention = intention;
//    }
//
//    @Override
//    public String toString() {
//        return  word + "\t" + intention;
//    }
//}